Found with m-c 20191212-ca62389e0be3. I will attach the test case once it is reduced.

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-divide-by-zero"

src/objdir-ff-ubsan/dist/include/mozilla/gfx/Matrix.h:693:52: runtime error: division by zero
    #0 0x7f2bb6a0aaef in mozilla::gfx::Point4DTyped<mozilla::gfx::UnknownUnits, float> mozilla::gfx::Matrix4x4Typed<mozilla::gfx::UnknownUnits, mozilla::gfx::UnknownUnits, float>::ProjectPoint<float>(mozilla::gfx::PointTyped<mozilla::gfx::UnknownUnits, float> const&) const src/objdir-ff-ubsan/dist/include/mozilla/gfx/Matrix.h:693:52
    #1 0x7f2bb6a0acdf in mozilla::gfx::RectTyped<mozilla::gfx::UnknownUnits, float> mozilla::gfx::Matrix4x4Typed<mozilla::gfx::UnknownUnits, mozilla::gfx::UnknownUnits, float>::ProjectRectBounds<float>(mozilla::gfx::RectTyped<mozilla::gfx::UnknownUnits, float> const&, mozilla::gfx::RectTyped<mozilla::gfx::UnknownUnits, float> const&) const src/objdir-ff-ubsan/dist/include/mozilla/gfx/Matrix.h:731:17
    #2 0x7f2bb698df8a in nsLayoutUtils::TransformRect(nsIFrame*, nsIFrame*, nsRect&) src/layout/base/nsLayoutUtils.cpp:2777:20
    #3 0x7f2bb6fe649a in ProcessFrameInternal(nsIFrame*, nsDisplayListBuilder*, AnimatedGeometryRoot**, nsRect&, nsIFrame*, nsTArray<nsIFrame*>&, bool) src/layout/painting/RetainedDisplayListBuilder.cpp:1012:17
    #4 0x7f2bb6fe5d66 in RetainedDisplayListBuilder::ProcessFrame(nsIFrame*, nsDisplayListBuilder*, nsIFrame*, nsTArray<nsIFrame*>&, bool, nsRect*, AnimatedGeometryRoot**) src/layout/painting/RetainedDisplayListBuilder.cpp:1209:8
    #5 0x7f2bb6fe715d in RetainedDisplayListBuilder::ComputeRebuildRegion(nsTArray<nsIFrame*>&, nsRect*, AnimatedGeometryRoot**, nsTArray<nsIFrame*>&) src/layout/painting/RetainedDisplayListBuilder.cpp:1310:10
    #6 0x7f2bb6fe8046 in RetainedDisplayListBuilder::AttemptPartialUpdate(unsigned int, mozilla::DisplayListChecker*) src/layout/painting/RetainedDisplayListBuilder.cpp:1455:8
    #7 0x7f2bb6995486 in nsLayoutUtils::PaintFrame(gfxContext*, nsIFrame*, nsRegion const&, unsigned int, nsDisplayListBuilderMode, nsLayoutUtils::PaintFrameFlags) src/layout/base/nsLayoutUtils.cpp:3956:40
    #8 0x7f2bb68c354b in mozilla::PresShell::Paint(nsView*, nsRegion const&, mozilla::PaintFlags) src/layout/base/PresShell.cpp:6033:5
    #9 0x7f2bb63c5f6c in nsViewManager::ProcessPendingUpdatesPaint(nsIWidget*) src/view/nsViewManager.cpp:461:18
    #10 0x7f2bb63c5842 in nsViewManager::ProcessPendingUpdatesForView(nsView*, bool) src/view/nsViewManager.cpp:396:22
    #11 0x7f2bb63c7882 in nsViewManager::ProcessPendingUpdates() src/view/nsViewManager.cpp:1019:5
    #12 0x7f2bb6857583 in nsRefreshDriver::Tick(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:2178:11
    #13 0x7f2bb68665ae in mozilla::RefreshDriverTimer::TickRefreshDrivers(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp, nsTArray<RefPtr<nsRefreshDriver> >&) src/layout/base/nsRefreshDriver.cpp:351:7
    #14 0x7f2bb6866311 in mozilla::RefreshDriverTimer::Tick(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:368:5
    #15 0x7f2bb6864cf9 in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::TickRefreshDriver(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:740:16
    #16 0x7f2bb6864057 in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::NotifyVsync(mozilla::VsyncEvent const&) src/layout/base/nsRefreshDriver.cpp:635:9
    #17 0x7f2bb6f3d4c6 in mozilla::layout::VsyncChild::RecvNotify(mozilla::VsyncEvent const&) src/layout/ipc/VsyncChild.cpp:65:16
    #18 0x7f2baffba3f6 in mozilla::layout::PVsyncChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PVsyncChild.cpp:187:54
    #19 0x7f2baf99e8eb in mozilla::ipc::PBackgroundChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PBackgroundChild.cpp:5876:32
    #20 0x7f2baf151b5b in mozilla::ipc::MessageChannel::DispatchAsyncMessage(mozilla::ipc::ActorLifecycleProxy*, IPC::Message const&) src/ipc/glue/MessageChannel.cpp:2209:25
    #21 0x7f2baf14cff7 in mozilla::ipc::MessageChannel::DispatchMessage(IPC::Message&&) src/ipc/glue/MessageChannel.cpp:2131:9
    #22 0x7f2baf14eb73 in mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask&) src/ipc/glue/MessageChannel.cpp:1973:3
    #23 0x7f2baf14fab8 in mozilla::ipc::MessageChannel::MessageTask::Run() src/ipc/glue/MessageChannel.cpp:2004:13
    #24 0x7f2badd69ec4 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1240:14
    #25 0x7f2badd6fb5e in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:486:10
    #26 0x7f2bb5b50138 in bool mozilla::SpinEventLoopUntil<(mozilla::ProcessFailureBehavior)1, mozilla::dom::ContentChild::ProvideWindowCommon(mozilla::dom::BrowserChild*, mozIDOMWindowProxy*, bool, unsigned int, bool, bool, bool, nsIURI*, nsTSubstring<char16_t> const&, nsTSubstring<char> const&, bool, bool, nsDocShellLoadState*, bool*, mozilla::dom::BrowsingContext**)::$_4>(mozilla::dom::ContentChild::ProvideWindowCommon(mozilla::dom::BrowserChild*, mozIDOMWindowProxy*, bool, unsigned int, bool, bool, bool, nsIURI*, nsTSubstring<char16_t> const&, nsTSubstring<char> const&, bool, bool, nsDocShellLoadState*, bool*, mozilla::dom::BrowsingContext**)::$_4&&, nsIThread*) src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:348:25
    #27 0x7f2bb5b4d6ee in mozilla::dom::ContentChild::ProvideWindowCommon(mozilla::dom::BrowserChild*, mozIDOMWindowProxy*, bool, unsigned int, bool, bool, bool, nsIURI*, nsTSubstring<char16_t> const&, nsTSubstring<char> const&, bool, bool, nsDocShellLoadState*, bool*, mozilla::dom::BrowsingContext**) src/dom/ipc/ContentChild.cpp:1251:5
    #28 0x7f2bb5bd739e in mozilla::dom::BrowserChild::ProvideWindow(mozIDOMWindowProxy*, unsigned int, bool, bool, bool, nsIURI*, nsTSubstring<char16_t> const&, nsTSubstring<char> const&, bool, bool, nsDocShellLoadState*, bool*, mozilla::dom::BrowsingContext**) src/dom/ipc/BrowserChild.cpp:936:14
    #29 0x7f2bba4c7550 in nsWindowWatcher::OpenWindowInternal(mozIDOMWindowProxy*, char const*, char const*, char const*, bool, bool, bool, nsIArray*, bool, bool, bool, nsDocShellLoadState*, mozilla::dom::BrowsingContext**) src/toolkit/components/windowwatcher/nsWindowWatcher.cpp:804:24
    #30 0x7f2bba4ca9ad in nsWindowWatcher::OpenWindow2(mozIDOMWindowProxy*, char const*, char const*, char const*, bool, bool, bool, nsISupports*, bool, bool, bool, nsDocShellLoadState*, mozilla::dom::BrowsingContext**) src/toolkit/components/windowwatcher/nsWindowWatcher.cpp:375:10
    #31 0x7f2bb2072513 in nsGlobalWindowOuter::OpenInternal(nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, bool, bool, bool, bool, bool, nsIArray*, nsISupports*, nsDocShellLoadState*, bool, mozilla::dom::BrowsingContext**) src/dom/base/nsGlobalWindowOuter.cpp:7201:21
    #32 0x7f2bb207199c in nsGlobalWindowOuter::OpenJS(nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, mozilla::dom::BrowsingContext**) src/dom/base/nsGlobalWindowOuter.cpp:5741:10
    #33 0x7f2bb20717bf in nsGlobalWindowOuter::OpenOuter(nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, mozilla::ErrorResult&) src/dom/base/nsGlobalWindowOuter.cpp:5714:12
    #34 0x7f2bb20145a7 in nsGlobalWindowInner::Open(nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, nsTSubstring<char16_t> const&, mozilla::ErrorResult&) src/dom/base/nsGlobalWindowInner.cpp:3722:3
    #35 0x7f2bb3498cbc in mozilla::dom::Window_Binding::open(JSContext*, JS::Handle<JSObject*>, nsGlobalWindowInner*, JSJitMethodCallArgs const&) src/objdir-ff-ubsan/dom/bindings/WindowBinding.cpp:2625:59
    #36 0x7f2bb3c91cd0 in bool mozilla::dom::binding_detail::GenericMethod<mozilla::dom::binding_detail::MaybeCrossOriginObjectThisPolicy, mozilla::dom::binding_detail::ThrowExceptions>(JSContext*, unsigned int, JS::Value*) src/dom/bindings/BindingUtils.cpp:3151:13
    #37 0x7f2bba858270 in CallJSNative(JSContext*, bool (*)(JSContext*, unsigned int, JS::Value*), js::CallReason, JS::CallArgs const&) src/js/src/vm/Interpreter.cpp:457:13
    #38 0x7f2bba858270 in js::InternalCallOrConstruct(JSContext*, JS::CallArgs const&, js::MaybeConstruct, js::CallReason) src/js/src/vm/Interpreter.cpp:549:12
    #39 0x7f2bba85927a in InternalCall(JSContext*, js::AnyInvokeArgs const&, js::CallReason) src/js/src/vm/Interpreter.cpp:618:10
    #40 0x7f2bbbaa9b86 in js::jit::DoCallFallback(JSContext*, js::jit::BaselineFrame*, js::jit::ICCall_Fallback*, unsigned int, JS::Value*, JS::MutableHandle<JS::Value>) src/js/src/jit/BaselineIC.cpp:2941:10
    #41 0x19f752b42f57  (<unknown module>)

The priority flag is not set for this bug.
:jbonisteel, could you have a look please?

For more information, please visit auto_nag documentation.